{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import os.path\n",
    "import tensorflow as tf\n",
    "import helper\n",
    "import project_tests as tests\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 2\n",
    "LEARNING_RATE = 0.00023949513325777832\n",
    "KEEP_PROB = 0.49548463810034943"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_vgg(sess, vgg_path):\n",
    "    \"\"\"\n",
    "    Load Pretrained VGG Model into TensorFlow.\n",
    "    :param sess: TensorFlow Session\n",
    "    :param vgg_path: Path to vgg folder, containing \"variables/\" and \"saved_model.pb\"\n",
    "    :return: Tuple of Tensors from VGG model (image_input, keep_prob, layer3_out, layer4_out, layer7_out)\n",
    "    \"\"\"\n",
    "    vgg_tag = 'vgg16'\n",
    "    vgg_input_tensor_name = 'image_input:0'\n",
    "    vgg_keep_prob_tensor_name = 'keep_prob:0'\n",
    "    vgg_layer3_out_tensor_name = 'layer3_out:0'\n",
    "    vgg_layer4_out_tensor_name = 'layer4_out:0'\n",
    "    vgg_layer7_out_tensor_name = 'layer7_out:0'\n",
    "\n",
    "    model = tf.saved_model.loader.load(sess, [vgg_tag], vgg_path)\n",
    "\n",
    "    graph = tf.get_default_graph()\n",
    "\n",
    "    image_input = graph.get_tensor_by_name(vgg_input_tensor_name)\n",
    "    keep_prob = graph.get_tensor_by_name(vgg_keep_prob_tensor_name)\n",
    "    layer3 = graph.get_tensor_by_name(vgg_layer3_out_tensor_name)\n",
    "    layer4 = graph.get_tensor_by_name(vgg_layer4_out_tensor_name)\n",
    "    layer7 = graph.get_tensor_by_name(vgg_layer7_out_tensor_name)\n",
    "    \n",
    "    return image_input, keep_prob, layer3, layer4, layer7\n",
    "\n",
    "# tests.test_load_vgg(load_vgg, tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def layers(vgg_layer3_out, vgg_layer4_out, vgg_layer7_out, num_classes):\n",
    "    \"\"\"\n",
    "    Create the layers for a fully convolutional network.  Build skip-layers using the vgg layers.\n",
    "    :param vgg_layer7_out: TF Tensor for VGG Layer 3 output\n",
    "    :param vgg_layer4_out: TF Tensor for VGG Layer 4 output\n",
    "    :param vgg_layer3_out: TF Tensor for VGG Layer 7 output\n",
    "    :param num_classes: Number of classes to classify\n",
    "    :return: The Tensor for the last layer of output\n",
    "    \"\"\"\n",
    "    # TODO: Implement function\n",
    "    kernel_regularizer = tf.contrib.layers.l2_regularizer(1e-3)\n",
    "    kernel_initializer = tf.contrib.layers.xavier_initializer_conv2d()\n",
    "    \n",
    "    pool3_1x1 = tf.layers.conv2d(vgg_layer3_out, num_classes, kernel_size=1, \n",
    "                                 padding='same', \n",
    "                                 kernel_initializer=kernel_initializer, \n",
    "                                 kernel_regularizer=kernel_regularizer)\n",
    "    \n",
    "    pool4_1x1 = tf.layers.conv2d(vgg_layer4_out, num_classes, kernel_size=1, \n",
    "                                 padding='same', \n",
    "                                 kernel_initializer=kernel_initializer, \n",
    "                                 kernel_regularizer=kernel_regularizer)\n",
    "    \n",
    "    conv7_1x1 = tf.layers.conv2d(vgg_layer7_out, num_classes, kernel_size=1, \n",
    "                                 padding='same', \n",
    "                                 kernel_initializer=kernel_initializer, \n",
    "                                 kernel_regularizer=kernel_regularizer)\n",
    "    \n",
    "    # make prediction of segmentation\n",
    "    deconv7 = tf.layers.conv2d_transpose(conv7_1x1, num_classes, kernel_size=4, strides=2, padding='same',\n",
    "                                         kernel_initializer=kernel_initializer, \n",
    "                                         kernel_regularizer=kernel_regularizer)\n",
    "    \n",
    "    fuse1 = tf.add(deconv7, pool4_1x1)\n",
    "    deconv_fuse1 = tf.layers.conv2d_transpose(fuse1, num_classes, kernel_size=4, strides=2, padding='same',\n",
    "                                              kernel_initializer=kernel_initializer,\n",
    "                                              kernel_regularizer=kernel_regularizer)\n",
    "    \n",
    "    fuse2 = tf.add(deconv_fuse1, pool3_1x1)\n",
    "    \n",
    "    out = tf.layers.conv2d_transpose(fuse2, num_classes, kernel_size=16, strides=8, padding='same',\n",
    "                                     kernel_initializer=kernel_initializer, \n",
    "                                     kernel_regularizer=kernel_regularizer)\n",
    "    \n",
    "    return out\n",
    "# tests.test_layers(layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def optimize(nn_last_layer, correct_label, learning_rate, num_classes):\n",
    "    \"\"\"\n",
    "    Build the TensorFLow loss and optimizer operations.\n",
    "    :param nn_last_layer: TF Tensor of the last layer in the neural network\n",
    "    :param correct_label: TF Placeholder for the correct label image\n",
    "    :param learning_rate: TF Placeholder for the learning rate\n",
    "    :param num_classes: Number of classes to classify\n",
    "    :return: Tuple of (logits, train_op, cross_entropy_loss)\n",
    "    \"\"\"\n",
    "    # TODO: Implement function\n",
    "    logits = tf.reshape(nn_last_layer, (-1, num_classes))\n",
    "    cross_entropy_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=correct_label))\n",
    "#     train_op = tf.train.MomentumOptimizer(learning_rate=learning_rate, momentum=0.9).minimize(cross_entropy_loss)\n",
    "    train_op = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cross_entropy_loss)\n",
    "    \n",
    "    return logits, train_op, cross_entropy_loss\n",
    "# tests.test_optimize(optimize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_nn(sess, epochs, batch_size, get_batches_fn, train_op, cross_entropy_loss, input_image,\n",
    "             correct_label, keep_prob, learning_rate):\n",
    "    \"\"\"\n",
    "    Train neural network and print out the loss during training.\n",
    "    :param sess: TF Session\n",
    "    :param epochs: Number of epochs\n",
    "    :param batch_size: Batch size\n",
    "    :param get_batches_fn: Function to get batches of training data.  Call using get_batches_fn(batch_size)\n",
    "    :param train_op: TF Operation to train the neural network\n",
    "    :param cross_entropy_loss: TF Tensor for the amount of loss\n",
    "    :param input_image: TF Placeholder for input images\n",
    "    :param correct_label: TF Placeholder for label images\n",
    "    :param keep_prob: TF Placeholder for dropout keep probability\n",
    "    :param learning_rate: TF Placeholder for learning rate\n",
    "    \"\"\"\n",
    "    # TODO: Implement function\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    loss_per_epoch = []\n",
    "    for epoch in range(epochs):\n",
    "        losses, i = [], 0\n",
    "        for images, labels in get_batches_fn(batch_size):\n",
    "            i += 1\n",
    "            feed_dict = {input_image: images, \n",
    "                         correct_label: labels, \n",
    "                         keep_prob: KEEP_PROB, \n",
    "                         learning_rate: LEARNING_RATE}\n",
    "            _, loss = sess.run([train_op, cross_entropy_loss], feed_dict=feed_dict)\n",
    "            losses.append(loss)\n",
    "        \n",
    "        training_loss = sum(losses) / len(losses)\n",
    "        loss_per_epoch.append(training_loss)\n",
    "        print(\" [-] epoch: %d/%d, loss: %.5f\" % (epoch+1, epochs, training_loss))\n",
    "    return loss_per_epoch\n",
    "\n",
    "# tests.test_train_nn(train_nn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tests Passed\n",
      "INFO:tensorflow:Restoring parameters from b'./data/vgg/variables/variables'\n",
      " [-] epoch: 1/30, loss: 3.28263\n",
      " [-] epoch: 2/30, loss: 0.70379\n",
      " [-] epoch: 3/30, loss: 0.66155\n",
      " [-] epoch: 4/30, loss: 0.63797\n",
      " [-] epoch: 5/30, loss: 0.61965\n",
      " [-] epoch: 6/30, loss: 0.60346\n",
      " [-] epoch: 7/30, loss: 0.58569\n",
      " [-] epoch: 8/30, loss: 0.55632\n",
      " [-] epoch: 9/30, loss: 0.44355\n",
      " [-] epoch: 10/30, loss: 0.24563\n",
      " [-] epoch: 11/30, loss: 0.18292\n",
      " [-] epoch: 12/30, loss: 0.15251\n",
      " [-] epoch: 13/30, loss: 0.13415\n",
      " [-] epoch: 14/30, loss: 0.11705\n",
      " [-] epoch: 15/30, loss: 0.10362\n",
      " [-] epoch: 16/30, loss: 0.09700\n",
      " [-] epoch: 17/30, loss: 0.09713\n",
      " [-] epoch: 18/30, loss: 0.08529\n",
      " [-] epoch: 19/30, loss: 0.07571\n",
      " [-] epoch: 20/30, loss: 0.07254\n",
      " [-] epoch: 21/30, loss: 0.06490\n",
      " [-] epoch: 22/30, loss: 0.06402\n",
      " [-] epoch: 23/30, loss: 0.07886\n",
      " [-] epoch: 24/30, loss: 0.05911\n",
      " [-] epoch: 25/30, loss: 0.05304\n",
      " [-] epoch: 26/30, loss: 0.04957\n",
      " [-] epoch: 27/30, loss: 0.04735\n",
      " [-] epoch: 28/30, loss: 0.04736\n",
      " [-] epoch: 29/30, loss: 0.04830\n",
      " [-] epoch: 30/30, loss: 0.05039\n"
     ]
    }
   ],
   "source": [
    "num_classes = 2\n",
    "image_shape = (160, 576)\n",
    "data_dir = './data'\n",
    "runs_dir = './runs'\n",
    "tests.test_for_kitti_dataset(data_dir)\n",
    "\n",
    "# Download pretrained vgg model\n",
    "helper.maybe_download_pretrained_vgg(data_dir)\n",
    "\n",
    "# OPTIONAL: Train and Inference on the cityscapes dataset instead of the Kitti dataset.\n",
    "# You'll need a GPU with at least 10 teraFLOPS to train on.\n",
    "#  https://www.cityscapes-dataset.com/\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.85\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.InteractiveSession(config=config)\n",
    "\n",
    "# Path to vgg model\n",
    "vgg_path = os.path.join(data_dir, 'vgg')\n",
    "# Create function to get batches\n",
    "get_batches_fn = helper.gen_batch_function(os.path.join(data_dir, 'data_road/training'), image_shape)\n",
    "\n",
    "# OPTIONAL: Augment Images for better results\n",
    "#  https://datascience.stackexchange.com/questions/5224/how-to-prepare-augment-images-for-neural-network\n",
    "\n",
    "# TODO: Build NN using load_vgg, layers, and optimize function\n",
    "image_input, keep_prob, layer3, layer4, layer7 = load_vgg(sess, vgg_path)\n",
    "out = layers(layer3, layer4, layer7, num_classes)\n",
    "\n",
    "correct_label = tf.placeholder(tf.int32)\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "logits, train_op, cross_entropy_loss = optimize(out, correct_label, learning_rate, num_classes)\n",
    "\n",
    "# TODO: Train NN using the train_nn function\n",
    "epochs = 30\n",
    "batch_size = BATCH_SIZE\n",
    "loss_per_epoch = train_nn(sess, epochs, batch_size, get_batches_fn, train_op, cross_entropy_loss, image_input, correct_label, keep_prob, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Finished. Saving test images to: ./runs/1511773860.2436142\n"
     ]
    }
   ],
   "source": [
    "# TODO: Save inference data using helper.save_inference_samples\n",
    "#  helper.save_inference_samples(runs_dir, data_dir, sess, image_shape, logits, keep_prob, input_image)\n",
    "helper.save_inference_samples(runs_dir, data_dir, sess, image_shape, logits, keep_prob, image_input)\n",
    "\n",
    "# OPTIONAL: Apply the trained model to a video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xuc3HV97/HXZ657mU02yYYFciEE\nghSot6QIWDWxWFE50h5v2CreeqgeL6it11q1Fh9i22OtR6tHCyreoohWSlGrNhEVAiSREJJwiSFX\nQpLdzWWvc/2cP+a3k9nNbDKb3d/O7sz7+XjMg5nffHf2+8mQeef7+/3m8zN3R0REBCBS6wmIiMj0\noVAQEZEShYKIiJQoFEREpEShICIiJQoFEREpUSiIiEiJQkFEREoUCiIiUhKr9QTGq6Ojw5csWTJi\nW39/P62trbWZUAjqrR6ov5rqrR6ov5rqrR6YWE0bNmzocvf5pxo340JhyZIlrF+/fsS2tWvXsnLl\nytpMKAT1Vg/UX031Vg/UX031Vg9MrCYz21XNOO0+EhGREoWCiIiUKBRERKREoSAiIiUKBRERKZlx\nZx+djnU7urn13p3s7hlg8dwWrrt8CZctnVfraYmITDt1v1JYt6ObG+/cSldvhvmpJF29GW68cyvr\ndnTXemoiItNO3YfCrffupCURIxY1DhxL05yI0JKIceu9O2s8MxGR6afuQ2F3zwCtySjpXIGDx4bI\n5p3WZJTdPQO1npqIyLRT96GweG4L/ek80YgBkCs4/ek8i+e21HhmIiLTT92HwnWXL2EgkyOdzePu\n9A5lGcjkuO7yJbWemojItFP3oXDZ0nl85OqL6EglyeSdVDLGR66+SGcfiYhU0BCnpF62dB7PWNjO\nO769kVetWKRAEBEZQ92vFIY1xSOYGQOZXK2nIiIybTVMKJgZrcko/Zl8raciIjJtNUwoALQkYgyk\ntVIQERlLQ4VCa0IrBRGRk2moUGhJaqUgInIyDRUKWimIiJxcQ4VCSzJGv1YKIiJjaqhQaE1EGcgU\nv9ksIiInaqhQaEnEcHeGsoVaT0VEZFoKLRTMrMnM7jezTWa2xcz+rsKYpJl918y2m9l9ZrYkrPkA\ntCajAPTrC2wiIhWFuVJIAy9092cAzwSuMrPLRo15C3DY3c8H/hn4dIjzoSVRDIWBtA42i4hUEloo\neFFf8DAe3EbvzL8G+Hpw//vAH5mZhTWnlkSx1ZNWCiIilYV6TMHMomb2IHAQ+Jm73zdqyAJgD4C7\n54CjQGjd6lqDUFD/IxGRymwqzsQxs3bgh8A73f3hsu1bgBe7+97g8e+AS929e9TPXw9cD9DZ2bl8\n9erVI16/r6+PVCp1ynkcSzs3P5zmRefEuaQjOsGqwlNtPTNJvdVUb/VA/dVUb/XAxGpatWrVBndf\nccqB7j4lN+BjwF+P2vZT4PLgfgzoIgiqsW7Lly/30dasWXPCtkoG0jl/81fv9x9v3l/V+Fqptp6Z\npN5qqrd63Ouvpnqrx31iNQHrvYrP6jDPPpofrBAws2bgSuCRUcPuAN4Q3H8l8N/B5EOh9tkiIicX\n5kV2zgK+bmZRiscuvufud5rZJygm1h3AzcA3zGw70ANcG+J8MDNa1OpCRGRMoYWCuz8EPKvC9o+W\n3R8CXhXWHCppTUbVFE9EZAwN9Y1mKJ6WqpWCiEhlDRcKrQmtFERExtJwodCS1EpBRGQsDRcKrcmY\nzj4SERlD44VCIkp/Wu2zRUQqabhQGG6fnc6pfbaIyGgNFwql9tk62CwicoKGC4VS+2wdbBYROUED\nhkLx+3p9WimIiJyg4UJB7bNFRMbWcKHQktTuIxGRsTRcKAyvFPp1SU4RkRM0XCiofbaIyNgaLhTU\nPltEZGwNFwqg9tkiImNpyFBQ+2wRkcoaMhTUPltEpLKGDAW1zxYRqawhQ6E1EdXZRyIiFTRkKLQk\nYmqfLSJSQUOGQmsyqvbZIiIVhBYKZrbIzNaY2TYz22JmN1QYs9LMjprZg8Hto2HNp1xL6VvN2oUk\nIlIuFuJr54C/cveNZtYGbDCzn7n71lHjfuXuV4c4jxO0lvU/mjeVv1hEZJoLbaXg7vvdfWNwvxfY\nBiwI6/eNR2mloIPNIiIjTMkxBTNbAjwLuK/C05eb2SYz+7GZXTwV81FTPBGRyizsM3DMLAX8Evik\nu/9g1HOzgIK795nZS4F/cfdlFV7jeuB6gM7OzuWrV68e8XxfXx+pVKrqOR1LOzc/nOZF58S5pCM6\n7prCNt56ZoJ6q6ne6oH6q6ne6oGJ1bRq1aoN7r7ilAPdPbQbEAd+Cry3yvE7gY6TjVm+fLmPtmbN\nmhO2ncxAOudv/ur9/uPN+8f1c1NlvPXMBPVWU73V415/NdVbPe4TqwlY71V8Dod59pEBNwPb3P0z\nY4w5MxiHmV1KcXdWd1hzGqb22SIilYV59tFzgdcDm83swWDbh4HFAO7+JeCVwNvMLAcMAtcGiRYq\ntc8WEakstFBw918Ddooxnwc+H9YcTkbts0VETtSQ32gGtc8WEamkYUNB7bNFRE7UsKGg9tkiIidq\n2FBQ+2wRkRM1bCiofbaIyIkaNhTUPltE5EQNGwpqny0icqKGDYXy9tkiIlLUsKGg9tkiIidq2FBQ\n+2wRkRM1bCi0lHYfaaUgIjKscUMhUQwFrRRERI5r2FBojkcx00pBRKRcw4ZCsX22Wl2IiJRr2FAA\ntc8WERmtoUOhJRHT9xRERMo0dCioKZ6IyEgNHQotyRh9OvtIRKSkoUNBKwURkZEaOhTUPltEZKSG\nDgW1zxYRGamhQ0Hts0VERgotFMxskZmtMbNtZrbFzG6oMMbM7HNmtt3MHjKzZ4c1n0rUPltEZKRY\niK+dA/7K3TeaWRuwwcx+5u5by8a8BFgW3J4DfDH475RQ+2wRkZFCWym4+3533xjc7wW2AQtGDbsG\nuNWL1gHtZnZWWHMaTe2zRURGmpJjCma2BHgWcN+opxYAe8oe7+XE4AiN2meLiIwU5u4jAMwsBdwO\nvNvdj41+usKPnHB+qJldD1wP0NnZydq1a0c839fXd8K2aqRzTld3mvs3HiW/L/Q/iqqdbj3TWb3V\nVG/1QP3VVG/1wBTV5O6h3YA48FPgvWM8//+A15Y9fhQ462SvuXz5ch9tzZo1J2yrRqFQ8Ld87X6/\nfcOe0/r5sJxuPdNZvdVUb/W4119N9VaP+8RqAtZ7FZ/bYZ59ZMDNwDZ3/8wYw+4ArgvOQroMOOru\n+8OaU4U5qn22iEiZMPeZPBd4PbDZzB4Mtn0YWAzg7l8C7gJeCmwHBoA3hTifitQ+W0TkuNBCwd1/\nTeVjBuVjHHh7WHOohtpni4gc19DfaIbitZp19pGISJFCQccURERKGj4UdExBROS4hg+F4ZWCq322\niIhCIZWMUiiofbaICCgUSk3xdAaSiEiVoWBm55lZMri/0szeZWbt4U5tagy3z9Y1FUREql8p3A7k\nzex8it9SPhf4dmizmkJqny0icly1oVBw9xzwp8Bn3f09wJS1uA6T2meLiBxXbShkzey1wBuAO4Nt\n8XCmNLXUPltE5LhqQ+FNwOXAJ939CTM7F/hmeNOaOi2J4WMKWimIiFTV+8iLl9B8F4CZzQHa3P2m\nMCc2VZrjUcy0UhARgerPPlprZrPMbC6wCfiqmY3VDntGUftsEZHjqt19NNuLV037n8BX3X05cGV4\n05paanUhIlJUbSjEzOws4NUcP9BcN9Q+W0SkqNpQ+ATFy2r+zt0fMLOlwOPhTWtqqX22iEhRtQea\nbwNuK3u8A3hFWJOaai2JGN39mVpPQ0Sk5qo90LzQzH5oZgfN7ICZ3W5mC8Oe3FTRMQURkaJqdx99\nFbgDOBtYAPxHsK0uqH22iEhRtaEw392/6u654PY1YH6I85pSrQm1zxYRgepDocvMXmdm0eD2OqA7\nzIlNpZak2meLiED1ofBmiqejPgXsB15JsfVFXWhNqH22iAhUGQruvtvdX+7u8939DHf/E4pfZBuT\nmd0SHJh+eIznV5rZUTN7MLh99DTmPyl0oR0RkaKJXHntvad4/mvAVacY8yt3f2Zw+8QE5jIhpQvt\n6LsKItLgJhIKdrIn3f1uoGcCrz9lWoePKahTqog0ODvd0zDNbLe7Lz7FmCXAne5+SYXnVlK8otte\n4Engr919yxivcz1wPUBnZ+fy1atXj3i+r6+PVCo1/iIC6Zzzr5vSPH9hjOWdVX2fL1QTrWc6qrea\n6q0eqL+a6q0emFhNq1at2uDuK0450N3HvAG9wLEKt14gd7KfDX5+CfDwGM/NAlLB/ZcCj5/q9dyd\n5cuX+2hr1qw5Ydt4FAoFf8vX7vfbN+yZ0OtMlonWMx3VW031Vo97/dVUb/W4T6wmYL1X8Rl70t1H\n7t7m7rMq3NrcfUL/pHb3Y+7eF9y/C4ibWcdEXvN0qX22iEjRRI4pTIiZnWlmFty/NJhLzb77oFYX\nIiJVNsQ7HWb2HWAl0GFme4GPEVzX2d2/RPG7Dm8zsxwwCFwbLHFqQu2zRURCDAV3f+0pnv888Pmw\nfv94qX22iEgNdx9NNzqmICKiUCjRMQUREYVCidpni4goFErUPltERKFQovbZIiIKhRK1zxYRUSiU\nqH22iIhCoUTts0VEFAolpZWC2meLSANTKAS0UhARUSiUNMejmKFWFyLS0BQKATOjORGjX7uPRKSB\nKRTKtKopnog0OIVCmRatFESkwSkUyrQmtVIQkcamUCij9tki0ugUCmXUPltEGp1CoYzaZ4tIo1Mo\nlFH7bBFpdAqFMmqfLSKNTqFQRu2zRaTRhRYKZnaLmR00s4fHeN7M7HNmtt3MHjKzZ4c1l2qpfbaI\nNLowVwpfA646yfMvAZYFt+uBL4Y4l6qoKZ6INLrQQsHd7wZ6TjLkGuBWL1oHtJvZWWHNpxpqny0i\nja6WxxQWAHvKHu8NttWMVgoi0uhiNfzdVmFbxS8ImNn1FHcx0dnZydq1a0c839fXd8K20+HudHen\n2bDpKMlDj0z49U7XZNUzndRbTfVWD9RfTfVWD0xNTbUMhb3AorLHC4EnKw109y8DXwZYsWKFr1y5\ncsTza9euZfS20/WDp37L4nPnsvKycybl9U7HZNYzXdRbTfVWD9RfTfVWD0xNTbXcfXQHcF1wFtJl\nwFF331/D+QBqny0ijS20lYKZfQdYCXSY2V7gY0AcwN2/BNwFvBTYDgwAbwprLuOh9tki0shCCwV3\nf+0pnnfg7WH9/tOl9tki0sj0jeZR1D5bRBqZQmEUtc8WkUamUBilJRFjQO2zRaRBKRRGaUlEyat9\ntog0KIXCKK1qny0iDUyhMIraZ4tII1MojKL22SLSyBQKo6gpnog0MoXCKGqfLSKNTKEwilYKItLI\natkldVp6aM8RNu87yqMHevnRg/u47vIlXLZ0Xq2nJSIyJbRSKLNuRzc3/uc2coUCLfEoXb0Zbrxz\nK+t2dNd6aiIiU0IrhTK33ruTlkSMpniUY0M5zIxIBG7+9Y4TVgvrdnRz67072d0zwOK5LVpRiEhd\nUCiU2d0zwPxUkrNmN9Pdl+bwQIZ8vsDenkH+5oebedqZbTyts43eoSyf/fnjtCRizE8lSyuKj1x9\nkYJBRGY0hUKZxXNb6OrN0N4cp705TsGdrr408WiE+W1J7nuih18+eojNe4/i5sxpSZDNx0jGIyTi\nEb5+z86KoaBVhYjMFAqFMtddvoQb79wKFM9C6k/ncYf3X3Uhly2dR6Hg7O4Z4Lpb7idqcHQwS09f\nBihe33n7gT4+fscW5rclOaMtyfy2JAeODvHVe3Yyqyl+ylXFcHhs2TXIxfs2KDxEZMopFMpctnQe\nH7n6olH/qr+g9MEciRhLOlq5ZMEsunozpJpipHN5MrkCRwayJGMR2lvi7DsyyKY9R8gXnM17j5LJ\n5+nqS5OIRohHIxTc+eR/buOdLzyfOa0J5rQk2HGol8/9YjstiRizE5xyl5RWHyISBoXCKJctnXfK\nD9fRK4pszknGIiM+wAsFp2cgw7VfXkdroolM3snm8mQLTi5fYFd3P999YE/pNYfDoykeI5d12khT\nKBT41F3buOHKZcxujjO7Oc6spjiPPHWMf/jJo1Ud01B4iMh4KBROw6lWFFBcVXSkklzQmaKrN0NH\n6vgfde9QjnmpBP/4qmdwuD/DkYEs7/zORjoSCXIF52guTTZfIJsrsKOrn2+t2z3i92/ee5RcoUBT\nIko8EiEeNfIF5x9+8ggfuOpC5rQmaG+O89iBXm768SM6IC4iVVMonKZqVhRQ+TjFQCbHe6+4gFQy\nRioZY9FcuOjs4i6ptqYYzYVBOua1lcLj0698OkcHshwdLN4+ePtDzEnEyRWcXMEZyObJZAt09WX4\n8t07Sr97896j5LxASyJGPBqhKRbBHb64djvPXNROUzw6Yq5aVYiIQiFk1awqYGR4uDu9Q7lSeMxq\nKu42WhSM/f2Fs0sBMqx3KEd7S5yPvfxiDvdnSuHRHi+GRyZfoG8oS6Hg7DsyyNu/tZGOVJIFc5pZ\n0N7M0cEMqx/Yw+wqDoiLSP1SKEyBalYV5eGxtQcWtiUqhgecZPXxxxewoL34IQ8nhocDh/szNMWj\nXPOsBew7PMiTRwbZvO8om3YfIZPP09OfYXZzPHiNGLfeW/k0WxGpT6GGgpldBfwLEAX+zd1vGvX8\nG4F/BPYFmz7v7v8W5pyms+HwWLt2LStXLj/puPGuPobDI5sv8IGXXDhibC5f4GWf+xUtiWaGssUz\npfrSOc6a3cTunoFQahWR6Sm0UDCzKPAF4EXAXuABM7vD3beOGvpdd39HWPOoV+NdfZwsPGLRCOed\nUTwgPrc1wdxUkj09A+zs6md+WxM9/cXtIlL/wlwpXApsd/cdAGa2GrgGGB0KEqLTPSB+5qwmDJjT\nEudvf/Qwr16xiOcv68DMQp6xiNSSuXs4L2z2SuAqd/+L4PHrgeeUrwqC3UefAg4BjwHvcfc9FV7r\neuB6gM7OzuWrV68e8XxfXx+pVCqUOmqhVvU80p3j57tzdA06Hc3GlYtjnJmK8PNdOfb0FljcFuHK\nc+LMTo4/GPQeTX/1VlO91QMTq2nVqlUb3H3FqcaFGQqvAl48KhQudfd3lo2ZB/S5e9rM3gq82t1f\neLLXXbFiha9fv37EtuI++JWTXULNTLd63J1fPnaI29bvpeDOJWfP4uEnj7HncPWnrk63miaq3uqB\n+qup3uqBidVkZlWFQpjXU9gLpbMoARYCT5YPcPdud08HD78CjH10VWrGzFj5tDP4xDUX0xKP8qVf\n7uC3u48wqymua06I1JkwQ+EBYJmZnWtmCeBa4I7yAWZ2VtnDlwPbQpyPTNC8VJKBbJ4F7c0U3Hn8\nQB+RSPG61rfeu7PW0xORSRBaKLh7DngH8FOKH/bfc/ctZvYJM3t5MOxdZrbFzDYB7wLeGNZ8ZHLs\nOTzAWe1NXHhmG/GY8URXP/Go6dRVkToR6vcU3P0u4K5R2z5adv9DwIfCnINMruFrTrQ1xVjakeLx\ng708drCXZy1qr/XURGQS6BrNMi7XXb6EgUyO3qEc8ahxRluSdLZAMhYhkyvUenoiMkEKBRmX4S/E\ndbQlONSXZtHcFt77ogsYyBS45TdPENbZbCIyNdT7SMat0hfizpzdxG3r99KRSvLK5QtrNDMRmSiF\ngkyKF198Jof6Mvx48346UglWPu2MWk9JRE6DQkEmhZnxZ5cuprsvzTfX7WZea5LfXzi71tMSkXHS\nMQWZNNGI8dYXnMfCOc188Zfb2d2t01RFZhqFgkyqpniUd1+5jJZEjM/+4jF6+jO1npKIjIN2H8mk\na29JcMMfLeOmHz/Ch27fBBiP7B3k4n0bdIlPkWlOKwUJxaK5LTxv2Tzu2dHD5n1HmR139UkSmQEU\nChKaux/v4uzZzWRyBfYPQDqfJxmPqk+SyDSmUJDQ7O4Z4Oz2JhbObaHgsK9nkJ1dfWzYdZiNuw+T\ny+sb0CLTjY4pSGiG+yTNa03gQ0bLrDb2Hxkkkyvwhf/eTmsyxh+cO5crzpvHoWND3LpuV9llQ3Xs\nQaQWFAoSmvJLfLpDPu+kkjE+/Ke/R6opxr2/6+Y3j3fxgw172Nk9wLzWBJ2zmkrHHj5y9UUVg2Hd\nju5R151WgIhMFu0+ktCU90k6loGOtgQfufoirji/g6cvbOcvX3Aen3nNM4hFIzTFoxwZyPLoU73s\n6umnuz/D39+5lbWPHmT7wT6GsnmgGAg33rmVrt4M81NJHbwWmWRaKUiohvskFS8jeOKF9VoSMQaz\neS48s41cwTk2mGUwm2cwk2dPzwDfuHdXaWxHKsnG3YfJF5xkPMpQrkBTIgIUL/IzerWgFYXI+CkU\npObKr9HQkUoC0DuU45IFs/n7P7mEvYcH2Xt4gL2HB/nZ1qcwg2OD2dLPO87jB+FTd22jvSXBnJY4\nB48N8cMH95FKxpjdFOfQsfSk7JIaHrdll753IfVJoSA1V37soTUZpT+dZyCT471XXEBHKklHKskz\ng4v4PLT3CIeOpYkH12/I5gscG8qRjEWIRY09hwd4aG+GDTsPk8nn6R3MsZ8hAAo47//+Jq555gJm\nN8dpb0kwuznOvsMDfOu+3aSSMea2JsY8pjG866olEWN2gkk79qEVjUwnCgWpueFjDyM/GC+o+ME4\nHCBmRltTjP50ntaEj/hgdnde9rlf0d4cJ1twsnknly+QyRU4OpRlKJvnqaNDHB3Mki84m/ceJZPP\nc7i/eIjNDAoOH7j9If7H088m1RQjlYxx24Y9ZHIFEjFnKA8tkWK/py/fvYMLOttIxCIkohHiUeO+\nJ3pKAVJ+7ONkQXOyceXjJztoarn6USBOPwoFmRYqXaNhrHGnChAzY0lHK129GWY1xUvbe4dynN+Z\n4m9edhFQDI++dI5XfPEeZjU1ky9ArlAgV3CyueIKJJMvsKt7gL50jh0H+0lEjZ7+DJlsgZ5MH+7O\njkP9vO+2TSPm+fCTR8nmCyRj0dK2bL7AX9+2iSvOOz7Xe37XzVA2TzwawSiGTL7gfOxHD/Pnl51D\nayJGazJGazLKE4f6+ca6XbQkorQ3xzk4xi6x8QTNeFY/pxM0JxsbViDKxCgUZMapJkDG2iV13eUX\nlMYUVxtxzj8jRVdvhtmtx/869A7luODMNj780t8rbXvrNzdwsHeI5liU7p7DzJrdSl86R1tTnDc+\ndwnpbIFMsCJ55KleZjfFKb8OXTGE8lzQ2Vb6/Xc/3kV7cxwzw3GK3+crcLA3zcZdh+nP5CkUiq8y\nvKKJRY6fNJgrFHj36t9yxfkdNMejNMejrHn0IOlcgXSuwLEhw4ChXIFP3bWNtzxvKVEzohGIWHGV\nk8kXiOULDOQggYPBv67ZzqK5LSRjEZKxCJv2HOGmHz8y7qAZHvt3/7GVt71gKeef0UbvUJZjQzn+\n7y8ep3coy0Amz8FeJx6NkC84//Rfj/Kxqy9mTmucea1JmhPRcQddtSufMFdeM3U1ZzPt8okrVqzw\n9evXj9hWPLNlZW0mFIJ6qwdqU9N4/nIOf+CUB8jJ/gU+1HeEplR7xXEA//tbG0oHz4f1DuXoaEvw\nr3++vOpx7k46V6AvnePPvrKO2c1xCg75glMoOLlCgaODWf7ieUsZyhYYzOb59n27SEQNp7jqAMgX\nCmTyzqVL5o6Y5/1P9JCIGmZGJpshEU/g7sWx5x4fOxxIiVgUMzCMbL5AUzzC5ed1AGAUd73d87tu\n0tk88VjxQz5fKO6+S0SjI66xcf/OHlriUeKxCBGDbN7J5PJkciN/d1Miyua9R8kXnNZklGikOD6d\nyzO7KcENVy4jHo2QiBmPPtXLLb/ZSXM8Qnagl6bWWQzlCrznRct4zrnziEasdFv/RA+fCoKu2vf9\nZONO9/+lal+zOR4l3X+EptScMceejJltcPcVpxoX6krBzK4C/gWIAv/m7jeNej4J3AosB7qB17j7\nzjDnJI1jMndJjR63tQcWtiVOeewDxl6pVDPOzGiKR2mKR0srmvbmkQGypKOVNz333NK27Qd7xwya\nL/zZs8kFH9QFd25Y/Vu6+zK0JmN09/Qwe3YbfeksbU1x3rryPNLZAulcno8f2MK8tgTuUAj+ITm8\n8rn47FkE2YO7c/djXcxqjmMGUTNi0QhRg750nvdfdSFtTTFmNcd5322b6O47cZ6zmmN88CUX0tOf\npac/TU9/lgeC8Bo+DoQXf9eBo2m+fs/O0s+Xr6YyWSeRGSRXKPC3P9xywkWfSmOjEQzDDHL5Au/5\n7oM859y5RCJGxIx1O7pJ547v4oPirsD3f38Tz79g/ojXvPuxQ6XdgV429n23beK553fgXjxb7jfb\ni685vOpznGy+wHu++1uevXguBffgBpt2HynV1ByBCzqKf16VTsOeDKGFgplFgS8ALwL2Ag+Y2R3u\nvrVs2FuAw+5+vpldC3waeE1YcxIZy3gC5GTfuygfN96gqfYgO5x+0JgZ8agRDw51vOUPl3LjnVuJ\nRQrErLgCcYd3/dEy/qBsVXH7xr0Vg2ZZZ2JEIAFs3H244tiFc1t42pltpW1vuGKMs87++ALOP6Nt\nxGv+evuhEa/pFE9Lntsa56ZXPKN0Jtqbv/YA7S0tABw+kmPWrBYK7hwZyPK6y88hn3fyXgzFxw70\n0tGWwDGgWLe705vO8exz5lAoFD+Uf/14F7OaYlCKBEjGIvRn8ixobxkxz3SuQCoZw+z42OZg7Hnz\nUwxvHt5tGIkcH4dDXybH85Z1ELFiSEXM2Lb/GB1tCSJmZAdzpT+v3T3hXMQqzJXCpcB2d98BYGar\ngWuA8lC4Bvh4cP/7wOfNzHym7dMSqWC8QVPNuMkOmmpXP9UG0njGTjQQh7J53vyHT2Nua6I0blln\nqhQembgxpyVB71COi85uZtWo64bf+dCTFcPrvDNSXHf5ktK29bt6Ko67sC3B21aeN+I1N+87UnHs\n09oS/K/nLy1tGys4z+9Mce2li0e85i8eOVAa25XtB6A/nWfx3JGBNFnCDIUFwJ6yx3uB54w1xt1z\nZnYUmAd0hTgvkRlrsoOmfOzJVj+nGzTVjJ3MQBzZb8vpHcpNOLzCCMTTfc1T1TQZQjvQbGavAl7s\n7n8RPH49cKm7v7NszJZgzN7OMYfIAAAGoUlEQVTg8e+CMd2jXut64HqAzs7O5atXrx7xu/r6+kil\nUqHUUQv1Vg/UX031Vg/UT02PdOf4+e4cB/pydKZiXLk4xoXzKv/7d3hs16DT0Wxjjq12XNivWU1N\nY1m1alVVB5px91BuwOXAT8sefwj40KgxPwUuD+7HKK4Q7GSvu3z5ch9tzZo1J2ybyeqtHvf6q6ne\n6nGvv5rqrR73idUErPcqPrvD7JL6ALDMzM41swRwLXDHqDF3AG8I7r8S+O9g8iIiUgOhHVPw4jGC\nd1BcDUSBW9x9i5l9gmJi3QHcDHzDzLYDPRSDQ0REaiTU7ym4+13AXaO2fbTs/hDwqjDnICIi1dNF\ndkREpEShICIiJTOu95GZHQJ2jdrcQX19t6He6oH6q6ne6oH6q6ne6oGJ1XSOu88/1aAZFwqVmNl6\nr+b82xmi3uqB+qup3uqB+qup3uqBqalJu49ERKREoSAiIiX1EgpfrvUEJlm91QP1V1O91QP1V1O9\n1QNTUFNdHFMQEZHJUS8rBRERmQQzOhTM7Coze9TMtpvZB2s9n8lgZjvNbLOZPWhm60/9E9OPmd1i\nZgfN7OGybXPN7Gdm9njw3zm1nON4jFHPx81sX/A+PWhmL63lHMfDzBaZ2Roz22ZmW8zshmD7TH6P\nxqppRr5PZtZkZveb2aagnr8Ltp9rZvcF79F3g75yk/u7Z+ruo+DKbo9RdmU34LU+8spuM46Z7QRW\nuPuMPb/azJ4P9AG3uvslwbZ/AHrc/aYgwOe4+wdqOc9qjVHPx4E+d/+nWs7tdJjZWcBZ7r7RzNqA\nDcCfAG9k5r5HY9X0ambg+2TFS7e1unufmcWBXwM3AO8FfuDuq83sS8Amd//iZP7umbxSKF3Zzd0z\nwPCV3aTG3P1uig0Oy10DfD24/3WKf2FnhDHqmbHcfb+7bwzu9wLbKF7waia/R2PVNCMF3a77gofx\n4ObACylepRJCeo9mcihUurLbjP2foIwD/2VmG4KLC9WLTnffD8W/wMAZpxg/E7zDzB4Kdi/NmF0t\n5cxsCfAs4D7q5D0aVRPM0PfJzKJm9iBwEPgZ8DvgiLvngiGhfObN5FCwCttm5r6wkZ7r7s8GXgK8\nPdh1IdPPF4HzgGcC+4H/U9vpjJ+ZpYDbgXe7+7Faz2cyVKhpxr5P7p5392cCCynuGfm9SsMm+/fO\n5FDYCywqe7wQeLJGc5k07v5k8N+DwA8p/s9QDw4E+32H9/8erPF8JsTdDwR/aQvAV5hh71Own/p2\n4Fvu/oNg84x+jyrVNNPfJwB3PwKsBS4D2s1s+JIHoXzmzeRQqObKbjOKmbUGB8kws1bgj4GHT/5T\nM0b5VfbeAPyohnOZsOEPz8CfMoPep+Ag5s3ANnf/TNlTM/Y9Gqummfo+mdl8M2sP7jcDV1I8TrKG\n4lUqIaT3aMaefQQQnF72WY5f2e2TNZ7ShJjZUoqrAyheAOnbM7EmM/sOsJJiR8cDwMeAfwe+BywG\ndgOvcvcZcfB2jHpWUtwl4cBO4C+H98dPd2b2h8CvgM1AIdj8YYr74GfqezRWTa9lBr5PZvZ0igeS\noxT/8f49d/9E8BmxGpgL/BZ4nbunJ/V3z+RQEBGRyTWTdx+JiMgkUyiIiEiJQkFEREoUCiIiUqJQ\nEBGREoWCSMDM8mXdNB+czM67ZrakvMuqyHQVO/UQkYYxGLQVEGlYWimInEJwjYtPB/3t7zez84Pt\n55jZL4Jma78ws8XB9k4z+2HQC3+TmV0RvFTUzL4S9Mf/r+CbqpjZu8xsa/A6q2tUpgigUBAp1zxq\n99Fryp475u6XAp+n+C16gvu3uvvTgW8Bnwu2fw74pbs/A3g2sCXYvgz4grtfDBwBXhFs/yDwrOB1\n3hpWcSLV0DeaRQJm1ufuqQrbdwIvdPcdQdO1p9x9npl1UbywSzbYvt/dO8zsELCwvP1A0M75Z+6+\nLHj8ASDu7jea2U8oXsTn34F/L+ujLzLltFIQqY6PcX+sMZWU96jJc/yY3suALwDLgQ1lXTBFppxC\nQaQ6ryn7773B/XsoducF+HOKl0wE+AXwNihdKGXWWC9qZhFgkbuvAd4PtAMnrFZEpor+RSJyXHNw\npathP3H34dNSk2Z2H8V/SL022PYu4BYzex9wCHhTsP0G4Mtm9haKK4K3UbzASyVR4JtmNpvihaP+\nOeifL1ITOqYgcgrBMYUV7t5V67mIhE27j0REpEQrBRERKdFKQUREShQKIiJSolAQEZEShYKIiJQo\nFEREpEShICIiJf8fGQ70+GpXQYsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fc184b862b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "ep = np.linspace(1,epochs,epochs)\n",
    "plt.plot(ep, loss_per_epoch, '-o', alpha=0.7)\n",
    "plt.grid()\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
